{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.4015109, -0.9158542, -0.5937396], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAc00lEQVR4nO3df3BTdb7/8VfS/OjPk9CWJlZa6XdBoF9+KAVKVhevtlLcrqsrO9d1GO26XB3ZwID1MloXcHR2tlyc0dVdRWecVf+4ilPvoiuCbi1QQGOBSqUU6Oou2l5oUmhJ0gJNf+R9/2B71kgF+iP9NPh6zGTWnvNJ8w5rniTnJNEgIgIiolFmVD0AEX0/MT5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpISy+LzwwguYOHEi4uPjkZ+fj71796oahYgUUBKft956C6WlpXjiiSfw2WefYdasWSgqKkJra6uKcYhIAYOKD5bm5+dj7ty5+OMf/wgACIfDyMrKwooVK/DYY49d8vrhcBgnTpxASkoKDAZDtMclosskIujo6EBmZiaMxos/tzGN0ky67u5u1NbWoqysTN9mNBpRWFgIj8cz4HVCoRBCoZD+8/Hjx5Gbmxv1WYloaJqbmzFhwoSLrhn1+Jw6dQp9fX1wOBwR2x0OB44ePTrgdcrLy/Hkk09esL25uRmapkVlTiIavGAwiKysLKSkpFxy7ajHZyjKyspQWlqq/9x/BzVNY3yIxqDLORwy6vFJT09HXFwcfD5fxHafzwen0zngdaxWK6xW62iMR0SjZNTPdlksFuTl5aGqqkrfFg6HUVVVBZfLNdrjEJEiSl52lZaWoqSkBHPmzMG8efPw+9//HmfOnMH999+vYhwiUkBJfO6++26cPHkS69atg9frxXXXXYcPPvjggoPQRHTlUvI+n+EKBoOw2WwIBAI84Ew0hgzmscnPdhGREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREowPESnB+BCREoOOz65du3D77bcjMzMTBoMB77zzTsR+EcG6detw1VVXISEhAYWFhfjiiy8i1rS3t2PJkiXQNA12ux1Lly5FZ2fnsO4IEcWWQcfnzJkzmDVrFl544YUB92/YsAHPP/88XnrpJdTU1CApKQlFRUXo6urS1yxZsgQNDQ2orKzEli1bsGvXLjz44INDvxdEFHtkGADI5s2b9Z/D4bA4nU55+umn9W1+v1+sVqu8+eabIiJy+PBhASD79u3T12zbtk0MBoMcP378sm43EAgIAAkEAsMZn4hG2GAemyN6zOfYsWPwer0oLCzUt9lsNuTn58Pj8QAAPB4P7HY75syZo68pLCyE0WhETU3NgL83FAohGAxGXIgoto1ofLxeLwDA4XBEbHc4HPo+r9eLjIyMiP0mkwmpqan6mm8rLy+HzWbTL1lZWSM5NhEpEBNnu8rKyhAIBPRLc3Oz6pGIaJhGND5OpxMA4PP5Irb7fD59n9PpRGtra8T+3t5etLe362u+zWq1QtO0iAsRxbYRjU9OTg6cTieqqqr0bcFgEDU1NXC5XAAAl8sFv9+P2tpafc327dsRDoeRn58/kuMQ0RhmGuwVOjs78eWXX+o/Hzt2DHV1dUhNTUV2djZWrVqF3/72t5g8eTJycnKwdu1aZGZm4s477wQATJs2DYsWLcIDDzyAl156CT09PVi+fDl+8YtfIDMzc8TuGBGNcYM9lbZjxw4BcMGlpKRERM6fbl+7dq04HA6xWq1SUFAgjY2NEb+jra1N7rnnHklOThZN0+T++++Xjo6Oy56Bp9qJxqbBPDYNIiIK2zckwWAQNpsNgUCAx3+IxpDBPDZj4mwXEV15GB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUoLxISIlGB8iUsKkegCiWCQigEj/D+f/13j+73KDwaBoqtgyqGc+5eXlmDt3LlJSUpCRkYE777wTjY2NEWu6urrgdruRlpaG5ORkLF68GD6fL2JNU1MTiouLkZiYiIyMDKxevRq9vb3DvzdEUSbhMLpPncLJrVvxjw0bcOyZZ3D0P/8Tf1uzBif++78ROn78fJjokgb1zKe6uhputxtz585Fb28vHn/8cSxcuBCHDx9GUlISAODhhx/G+++/j4qKCthsNixfvhx33XUXPv74YwBAX18fiouL4XQ68cknn6ClpQX33XcfzGYzfve73438PSQaJhGB9PUhdOIE2j76CO27d6Pn9GkgHI5Y19nQgPbqamQ98ABsc+bAYORRjYsxyDAyffLkSWRkZKC6uhoLFixAIBDA+PHj8cYbb+DnP/85AODo0aOYNm0aPB4P5s+fj23btuEnP/kJTpw4AYfDAQB46aWX8Oijj+LkyZOwWCyXvN1gMAibzYZAIABN04Y6PtEl9Z07hzNffIG27dsR3L8fvcHgJa9jTk3F/3v0USRPmzYKE44tg3lsDivNgUAAAJCamgoAqK2tRU9PDwoLC/U1U6dORXZ2NjweDwDA4/FgxowZengAoKioCMFgEA0NDQPeTigUQjAYjLgQRYuIoDcYRPuuXfjyt7/Fl08+ifbt2y8rPADQ096OlooKhEOhKE8a24Z8wDkcDmPVqlW44YYbMH36dACA1+uFxWKB3W6PWOtwOOD1evU13wxP//7+fQMpLy/Hk08+OdRRiS6p/wVAt88H/6efom3nTpz76qsLXlpdrs76evT4/bB+6991+pchx8ftduPQoUPYs2fPSM4zoLKyMpSWluo/B4NBZGVlRf126conIkA4jHNNTWjbsQOnd+9GT1vbyPxeuqghxWf58uXYsmULdu3ahQkTJujbnU4nuru74ff7I579+Hw+OJ1Ofc3evXsjfl//2bD+Nd9mtVphtVqHMirRgEQE0t2NjkOH0L5zJ4IHDlz2y6rLwYPNlzaoPyERwfLly7F582Zs374dOTk5Efvz8vJgNptRVVWlb2tsbERTUxNcLhcAwOVyob6+Hq2trfqayspKaJqG3Nzc4dwXoksSEfQEg2ivrsYXTz2Fv5eXo726ekTDAwDaddfBPG7ciP7OK82gnvm43W688cYbePfdd5GSkqIfo7HZbEhISIDNZsPSpUtRWlqK1NRUaJqGFStWwOVyYf78+QCAhQsXIjc3F/feey82bNgAr9eLNWvWwO1289kNRY309aH71Cm079yJ0x9/jK7//V9IlN5bZsnIgPPf/x3Gyzhz+302qFPt3/XOzVdffRW//OUvAZx/k+EjjzyCN998E6FQCEVFRXjxxRcjXlJ9/fXXWLZsGXbu3ImkpCSUlJRg/fr1MJkur4U81U6XK9zTg3Nff422HTvg//jj8+/PidbxGIMB8VdfjawHH0TKrFnfy3c6D+axOaz3+ajC+NDFiAjCoRDOHDmC1m3b0HnoEPo6O6N2ewazGfFZWbDn5yPtlltgycj4XoYHGNxjk5/toitC/9+hvYEAAnv3om3nTpw5ejRqL60AwBgfj5TrrkP6rbciZfp0GOPjAfCzXZeL8aGY1n+qPHTyJE7v3o22qiqETpyI3g0aDDCnpkKbPRvjFy1CwsSJMJrN0bu9KxjjQzFJRCA9PTj7j3+gbccOBPbtO//+nGgdRTAaEZ+ZidR/+zeMu/FGWB0OGOLionNb3xOMD8UUEUHf2bPoPHwYbZWVCH7+OcLnzkXt9oxWKxInT0bqggWwu1wwpaTwPTwjhPGhmCAi6D19Gqc9HrTv2oWzf/87pLs7arcXl5SElFmzkL5wIZKnTYMxPp7HckYY40NjVv8XdoVaWtC+axfaq6sRammJ3ksrAObx4zFu/nyMW7AAST/4ARAXx+hECeNDY07/9+ec/fJLnPzgg/MffTh9Ono3aDDAevXVSC8owLgbb4Rl/HjAYGB0oozxoTFDRBA+dw4d9fU49dFH6Dh4MKrHcwxmM5KnTUPaLbdAu/56mOx2BmcUMT6knITD6PH7EaipwckPPzz/0YcoHs8xJiRAu/56jF+0CElTpvB4jiKMDykT7uk5fzxn926c3r0bIa93yN+fc0kGAyzp6Rh3441IXbAA8VlZ/OyVYowPjToRQU9bG1reegun9+xB39mzUTuIbDCZEH/NNUj90Y+Q+qMfwZyaChiNfKYzBjA+NKpEBKETJ/DVc8/hzNGjUbsdg9mM5NxcpC9cCO266xCXnMzgjDGMD42qvrNn0fTyy+c/dyUCf3c3/t7RAZvFgh+kpMA4zECYNA3anDlIu/lmJOfmwmAyMTpjFONDoyqwbx86Dh6EiKDpzBmsPXAAjYEAkkwm/Me11+LunBzEDTYWBgPMaWlIXbAAaTffjPgJE/jRhxjA+NCo6jx8GAiHIQD+q74eh/1+AECwpwd/PHIE/99ux6x//tdQLsVgMiFh4kSk3nwz7PPmwTJ+PD/6EEMYH1Im2NMT8XN3OIxQX98lr2dMSEDStdcivajo/PGcpCS+tIpBjA+NqriEBACAAcDNTie+DAbR+88zXddqGq5JTv7O65rsdtjmzkXaLbcgadIkGCwWRieGMT40qtJuvhmnKivR19mJkkmTkGI246OWFlyVkIAHrr0WGf/8Qi6dwQBLRgbG3XAD0goKEJ+ZyVPlVwh+jSqNKgmH0fr++zj++uuQ7u7zn+PC+WdCwDe+BdBoROKkSUgvLIRt3jyYx41jcGIAv0aVxiyD0YjxixYBAHz/8z/oOX0ahv6//wwGGOPjkTRlyvn351x/PeISExmdKxTjQ6POaDYjo7gYttmzETxwACGvF8aEBBjNZqTMnInEnBwY+Z9RuuIxPqSEwWhE/NVXI/7qq1WPQorwTRFEpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKDCo+GzduxMyZM6FpGjRNg8vlwrZt2/T9XV1dcLvdSEtLQ3JyMhYvXgyfzxfxO5qamlBcXIzExERkZGRg9erV6O3tHZl7Q0QxY1DxmTBhAtavX4/a2lrs378ft9xyC+644w40NDQAAB5++GG89957qKioQHV1NU6cOIG77rpLv35fXx+Ki4vR3d2NTz75BK+//jpee+01rFu3bmTvFRGNfTJM48aNk1deeUX8fr+YzWapqKjQ9x05ckQAiMfjERGRrVu3itFoFK/Xq6/ZuHGjaJomoVDoO2+jq6tLAoGAfmlubhYAEggEhjs+EY2gQCBw2Y/NIR/z6evrw6ZNm3DmzBm4XC7U1taip6cHhYWF+pqpU6ciOzsbHo8HAODxeDBjxgw4HA59TVFREYLBoP7saSDl5eWw2Wz6JSsra6hjE9EYMej41NfXIzk5GVarFQ899BA2b96M3NxceL1eWCwW2O32iPUOhwNerxcA4PV6I8LTv79/33cpKytDIBDQL83NzYMdm4jGmEF/h/OUKVNQV1eHQCCAt99+GyUlJaiuro7GbDqr1Qorv1Cc6Ioy6PhYLBZMmjQJAJCXl4d9+/bhueeew913343u7m74/f6IZz8+nw9OpxMA4HQ6sXfv3ojf1382rH8NEX0/DPt9PuFwGKFQCHl5eTCbzaiqqtL3NTY2oqmpCS6XCwDgcrlQX1+P1tZWfU1lZSU0TUNubu5wRyGiGDKoZz5lZWW47bbbkJ2djY6ODrzxxhvYuXMnPvzwQ9hsNixduhSlpaVITU2FpmlYsWIFXC4X5s+fDwBYuHAhcnNzce+992LDhg3wer1Ys2YN3G43X1YRfc8MKj6tra2477770NLSApvNhpkzZ+LDDz/ErbfeCgB49tlnYTQasXjxYoRCIRQVFeHFF1/Urx8XF4ctW7Zg2bJlcLlcSEpKQklJCZ566qmRvVdENObxv9VORCNmMI9NfraLiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJRgfIhICcaHiJQYVnzWr18Pg8GAVatW6du6urrgdruRlpaG5ORkLF68GD6fL+J6TU1NKC4uRmJiIjIyMrB69Wr09vYOZxQiijFDjs++ffvw8ssvY+bMmRHbH374Ybz33nuoqKhAdXU1Tpw4gbvuukvf39fXh+LiYnR3d+OTTz7B66+/jtdeew3r1q0b+r0gotgjQ9DR0SGTJ0+WyspKuemmm2TlypUiIuL3+8VsNktFRYW+9siRIwJAPB6PiIhs3bpVjEajeL1efc3GjRtF0zQJhUID3l5XV5cEAgH90tzcLAAkEAgMZXwiipJAIHDZj80hPfNxu90oLi5GYWFhxPba2lr09PREbJ86dSqys7Ph8XgAAB6PBzNmzIDD4dDXFBUVIRgMoqGhYcDbKy8vh81m0y9ZWVlDGZuIxpBBx2fTpk347LPPUF5efsE+r9cLi8UCu90esd3hcMDr9eprvhme/v39+wZSVlaGQCCgX5qbmwc7NhGNMabBLG5ubsbKlStRWVmJ+Pj4aM10AavVCqvVOmq3R0TRN6hnPrW1tWhtbcXs2bNhMplgMplQXV2N559/HiaTCQ6HA93d3fD7/RHX8/l8cDqdAACn03nB2a/+n/vXENGVb1DxKSgoQH19Perq6vTLnDlzsGTJEv2fzWYzqqqq9Os0NjaiqakJLpcLAOByuVBfX4/W1lZ9TWVlJTRNQ25u7gjdLSIa6wb1sislJQXTp0+P2JaUlIS0tDR9+9KlS1FaWorU1FRomoYVK1bA5XJh/vz5AICFCxciNzcX9957LzZs2ACv14s1a9bA7XbzpRXR98ig4nM5nn32WRiNRixevBihUAhFRUV48cUX9f1xcXHYsmULli1bBpfLhaSkJJSUlOCpp54a6VGIaAwziIioHmKwgsEgbDYbAoEANE1TPQ4R/dNgHpv8bBcRKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKcH4EJESjA8RKWFSPcBQiAgAIBgMKp6EiL6p/zHZ/xi9mJiMT1tbGwAgKytL8SRENJCOjg7YbLaLronJ+KSmpgIAmpqaLnkHx5pgMIisrCw0NzdD0zTV41w2zj26YnVuEUFHRwcyMzMvuTYm42M0nj9UZbPZYur/mG/SNC0mZ+fcoysW577cJwQ84ExESjA+RKRETMbHarXiiSeegNVqVT3KoMXq7Jx7dMXq3INhkMs5J0ZENMJi8pkPEcU+xoeIlGB8iEgJxoeIlGB8iEiJmIzPCy+8gIkTJyI+Ph75+fnYu3ev0nl27dqF22+/HZmZmTAYDHjnnXci9osI1q1bh6uuugoJCQkoLCzEF198EbGmvb0dS5YsgaZpsNvtWLp0KTo7O6M6d3l5OebOnYuUlBRkZGTgzjvvRGNjY8Sarq4uuN1upKWlITk5GYsXL4bP54tY09TUhOLiYiQmJiIjIwOrV69Gb29v1ObeuHEjZs6cqb/71+VyYdu2bWN65oGsX78eBoMBq1atirnZR4TEmE2bNonFYpE//elP0tDQIA888IDY7Xbx+XzKZtq6dav85je/kT//+c8CQDZv3hyxf/369WKz2eSdd96Rzz//XH76059KTk6OnDt3Tl+zaNEimTVrlnz66aeye/dumTRpktxzzz1RnbuoqEheffVVOXTokNTV1cmPf/xjyc7Ols7OTn3NQw89JFlZWVJVVSX79++X+fPnyw9/+EN9f29vr0yfPl0KCwvlwIEDsnXrVklPT5eysrKozf2Xv/xF3n//ffnb3/4mjY2N8vjjj4vZbJZDhw6N2Zm/be/evTJx4kSZOXOmrFy5Ut8eC7OPlJiLz7x588Ttdus/9/X1SWZmppSXlyuc6l++HZ9wOCxOp1OefvppfZvf7xer1SpvvvmmiIgcPnxYAMi+ffv0Ndu2bRODwSDHjx8ftdlbW1sFgFRXV+tzms1mqaio0NccOXJEAIjH4xGR8+E1Go3i9Xr1NRs3bhRN0yQUCo3a7OPGjZNXXnklJmbu6OiQyZMnS2Vlpdx00016fGJh9pEUUy+7uru7UVtbi8LCQn2b0WhEYWEhPB6Pwsm+27Fjx+D1eiNmttlsyM/P12f2eDyw2+2YM2eOvqawsBBGoxE1NTWjNmsgEADwr28NqK2tRU9PT8TsU6dORXZ2dsTsM2bMgMPh0NcUFRUhGAyioaEh6jP39fVh06ZNOHPmDFwuV0zM7Ha7UVxcHDEjEBt/3iMppj7VfurUKfT19UX8wQOAw+HA0aNHFU11cV6vFwAGnLl/n9frRUZGRsR+k8mE1NRUfU20hcNhrFq1CjfccAOmT5+uz2WxWGC32y86+0D3rX9ftNTX18PlcqGrqwvJycnYvHkzcnNzUVdXN2ZnBoBNmzbhs88+w759+y7YN5b/vKMhpuJD0eN2u3Ho0CHs2bNH9SiXZcqUKairq0MgEMDbb7+NkpISVFdXqx7ropqbm7Fy5UpUVlYiPj5e9TjKxdTLrvT0dMTFxV1w9N/n88HpdCqa6uL657rYzE6nE62trRH7e3t70d7ePir3a/ny5diyZQt27NiBCRMm6NudTie6u7vh9/svOvtA961/X7RYLBZMmjQJeXl5KC8vx6xZs/Dcc8+N6Zlra2vR2tqK2bNnw2QywWQyobq6Gs8//zxMJhMcDseYnT0aYio+FosFeXl5qKqq0reFw2FUVVXB5XIpnOy75eTkwOl0RswcDAZRU1Ojz+xyueD3+1FbW6uv2b59O8LhMPLz86M2m4hg+fLl2Lx5M7Zv346cnJyI/Xl5eTCbzRGzNzY2oqmpKWL2+vr6iHhWVlZC0zTk5uZGbfZvC4fDCIVCY3rmgoIC1NfXo66uTr/MmTMHS5Ys0f95rM4eFaqPeA/Wpk2bxGq1ymuvvSaHDx+WBx98UOx2e8TR/9HW0dEhBw4ckAMHDggAeeaZZ+TAgQPy9ddfi8j5U+12u13effddOXjwoNxxxx0Dnmq//vrrpaamRvbs2SOTJ0+O+qn2ZcuWic1mk507d0pLS4t+OXv2rL7moYcekuzsbNm+fbvs379fXC6XuFwufX//qd+FCxdKXV2dfPDBBzJ+/Pionvp97LHHpLq6Wo4dOyYHDx6Uxx57TAwGg/z1r38dszN/l2+e7Yq12Ycr5uIjIvKHP/xBsrOzxWKxyLx58+TTTz9VOs+OHTsEwAWXkpISETl/un3t2rXicDjEarVKQUGBNDY2RvyOtrY2ueeeeyQ5OVk0TZP7779fOjo6ojr3QDMDkFdffVVfc+7cOfn1r38t48aNk8TERPnZz34mLS0tEb/nq6++kttuu00SEhIkPT1dHnnkEenp6Yna3L/61a/kmmuuEYvFIuPHj5eCggI9PGN15u/y7fjE0uzDxe/zISIlYuqYDxFdORgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJf4PQtj9NPzAKxcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1. -1. -8.], [1. 1. 8.], (3,), float32)\n",
      "env.action_space= Box(-2.0, 2.0, (1,), float32)\n",
      "state= [-0.6658319  -0.74610174  0.6986953 ]\n",
      "action= [0.21158014]\n",
      "next_state= [-0.6594339  -0.75176257  0.170856  ]\n",
      "reward= -5.336123894632103\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7f9dd4e773d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-1298.6173694218974, 316.6596681170837)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/3.A2C.Pendulum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b05f4766",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-272.44372607399566, 229.55795657113933)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/3.A2C.Pendulum')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAa5ElEQVR4nO3df2zT973v8ZedxE5I8nUIEJuMpEQXWoj4sRIgeJXa3eGRtllXRnZPhxDLGKcVzCAgExrZgGpoUrhUGisbhB1VA6Qjmp30FDpSaJcTaFiFCRDIGn5l3T2syQHsUFjswIid2O/7R5vvMKQ0zg9/7PB6SJbI9/tx/DbFz37tb+wYRERARBRlRtUDENGjifEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiUYHyJSgvEhIiWUxWfnzp2YOHEikpOTUVhYiFOnTqkahYgUUBKf3//+9ygrK8Orr76Ks2fPYubMmSgqKkJ7e7uKcYhIAYOKN5YWFhZizpw5+M1vfgMACIVCyMnJwerVq7Fhw4YvvX4oFMK1a9eQnp4Og8Ew3OMSUT+JCDo7O5GdnQ2j8eHHNolRmkkXCATQ2NiI8vJyfZvRaITD4YDL5erzOn6/H36/X//66tWryM/PH/ZZiWhg2traMGHChIeuiXp8Pv30UwSDQVit1rDtVqsVly9f7vM6FRUV+PnPf/7A9ra2NmiaNixzElHkfD4fcnJykJ6e/qVrox6fgSgvL0dZWZn+de8d1DSN8SGKQf15OSTq8Rk7diwSEhLg8XjCtns8Hthstj6vYzabYTabozEeEUVJ1M92mUwmFBQUoK6uTt8WCoVQV1cHu90e7XGISBElT7vKyspQWlqK2bNnY+7cufjVr36FO3fuYNmyZSrGISIFlMTnpZdewo0bN7B582a43W589atfxXvvvffAi9BENHIp+TmfwfL5fLBYLPB6vXzBmSiGRPLY5Hu7iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEgJxoeIlGB8iEiJiONz/PhxvPDCC8jOzobBYMDBgwfD9osINm/ejPHjxyMlJQUOhwMff/xx2Jpbt25hyZIl0DQNGRkZWL58OW7fvj2oO0JE8SXi+Ny5cwczZ87Ezp07+9y/bds27NixA7t370ZDQwNSU1NRVFSErq4ufc2SJUtw4cIF1NbWoqamBsePH8crr7wy8HtBRPFHBgGAHDhwQP86FAqJzWaT1157Td/W0dEhZrNZ3nzzTRERuXjxogCQ06dP62uOHDkiBoNBrl692q/b9Xq9AkC8Xu9gxieiIRbJY3NIX/O5cuUK3G43HA6Hvs1isaCwsBAulwsA4HK5kJGRgdmzZ+trHA4HjEYjGhoa+vy+fr8fPp8v7EJE8W1I4+N2uwEAVqs1bLvVatX3ud1uZGVlhe1PTExEZmamvuZ+FRUVsFgs+iUnJ2coxyYiBeLibFd5eTm8Xq9+aWtrUz0SEQ3SkMbHZrMBADweT9h2j8ej77PZbGhvbw/b39PTg1u3bulr7mc2m6FpWtiFiOLbkMYnLy8PNpsNdXV1+jafz4eGhgbY7XYAgN1uR0dHBxobG/U1R48eRSgUQmFh4VCOQ0QxLDHSK9y+fRt//etf9a+vXLmCpqYmZGZmIjc3F2vXrsUvfvELTJ48GXl5edi0aROys7OxcOFCAMDUqVPx7LPP4uWXX8bu3bvR3d2NVatW4Xvf+x6ys7OH7I4RUYyL9FTasWPHBMADl9LSUhH57HT7pk2bxGq1itlslvnz50tLS0vY97h586YsXrxY0tLSRNM0WbZsmXR2dvZ7Bp5qJ4pNkTw2DSIiCts3ID6fDxaLBV6vl6//EMWQSB6bcXG2i4hGHsaHiJRgfIhIiYjPdhENlogg2NkJv9uNkN+PhPR0JI8fD4PJBIPBoHo8ihLGh6IqFAjg5rFjuPHuu+i6fh0SCCAhNRWpjz+O8f/yL0idOpUBekQwPhQ1oUAA16ur4Xn7bUh3t749ePs2fGfP4u4nn+CxVaugzZrFAD0C+JoPRYWIoKOhAe0HD4aF517dN2+i7d/+DYH73n5DIxPjQ1ER8vvRfugQQn7/Q9f5r1/HzXvenkMjF+NDUSE9PfB/wUem3K/rf/5nmKehWMD4UMwREcThD95ThBgfij3BIBAKqZ6ChhnjQzFHgkEe+TwCGB+KOcIjn0cC40MxR3p6AB75jHiMD8UcCQYhPPIZ8RgfijkSDPLI5xHA+FDMkZ4eHvk8Ahgfijk9Ph9CgYDqMWiYMT4Uc7pv3ULo7l3VY9AwY3yISAnGh4iUYHyISAnGh6LCmJSE5AkTVI9BMYTxoegwGJCQmqp6CoohjA9Fh8EAg5H/3Oif+K+BosaQkKB6BIohjA9Fh8EQWXz49ooRj/GhqDAAQARPu6SnZ9hmodjA+FB0RHjkE2J8RjzGh6Imkvh80a/XoZGD8aGo6ffZLhE+7XoEMD4UHQYD0M8jHwGfdj0KGB+KGj7tonsxPhQdkfyQIZ92PRIYH4qaiM528chnxIsoPhUVFZgzZw7S09ORlZWFhQsXoqWlJWxNV1cXnE4nxowZg7S0NJSUlMDj8YStaW1tRXFxMUaNGoWsrCysX78ePfw/3cjX3yOfUAh3//u/h3cWUi6i+NTX18PpdOLkyZOora1Fd3c3FixYgDt37uhr1q1bh0OHDqG6uhr19fW4du0aFi1apO8PBoMoLi5GIBDAiRMnsG/fPuzduxebN28euntFMcdgMES0PuT3D9MkFCsMMohfDXnjxg1kZWWhvr4eTz/9NLxeL8aNG4f9+/fju9/9LgDg8uXLmDp1KlwuF+bNm4cjR47gW9/6Fq5duwar1QoA2L17N37yk5/gxo0bMJlMX3q7Pp8PFosFXq8XmqYNdHyKsqv//u9w/8d/9GvtuOefR+6KFcM8EQ21SB6bg3rNx+v1AgAyMzMBAI2Njeju7obD4dDXTJkyBbm5uXC5XAAAl8uF6dOn6+EBgKKiIvh8Ply4cKHP2/H7/fD5fGEXIopvA45PKBTC2rVr8dRTT2HatGkAALfbDZPJhIyMjLC1VqsVbrdbX3NveHr39+7rS0VFBSwWi37JyckZ6NhEFCMGHB+n04nz58+jqqpqKOfpU3l5Obxer35pa2sb9tskouGVOJArrVq1CjU1NTh+/Dgm3PPRmDabDYFAAB0dHWFHPx6PBzabTV9z6tSpsO/Xezasd839zGYzzGbzQEYlohgV0ZGPiGDVqlU4cOAAjh49iry8vLD9BQUFSEpKQl1dnb6tpaUFra2tsNvtAAC73Y7m5ma0t7fra2pra6FpGvLz8wdzX4gojkR05ON0OrF//3688847SE9P11+jsVgsSElJgcViwfLly1FWVobMzExomobVq1fDbrdj3rx5AIAFCxYgPz8fS5cuxbZt2+B2u7Fx40Y4nU4e3VAYEYn4FD3Fj4jiU1lZCQD4+te/HrZ9z549+MEPfgAA2L59O4xGI0pKSuD3+1FUVIRdu3bpaxMSElBTU4OVK1fCbrcjNTUVpaWl2LJly+DuCY0o/F3tI19E8enPjwQlJydj586d2Llz5xeueeyxx3D48OFIbpoeMdLT89lHqfLIZ8Tie7soJunxoRGL8aGYJD09/TrSpvjF+FDUJI0e3e83l0owyCOfEY7xoagxZWX1+2M1+LRr5GN8KGoi+iRDxmfEY3woaiKKTzDI13xGOMaHosZgNPb71HnP7dv8KNURjvGhqInkyMfvdiMUCAzjNKQa40PRk5DAHxokHeNDUWNISADTQ70YH4qafv/qHHok8F8DRY2BT7voHgP6MDGigbj/BWcRQUcggP/X2QmLyYT/lZ4OI+P0yGB8KHruiY+IoPXOHWw6dw4tXi9SExPxr48/jpfy8pDAAD0S+LSLouben/MRAP+3uRkXOzoQFIGvuxu/uXQJ5//+d7VDUtQwPhQ1hqQkGO/5tErffb8SORAKwR8MRnssUoTxoahJyshA5tNPAwAMAP63zYbEe55iPa5peCwtDQCQYDbz7NgIx9d8KGoMRiPGOhy4efQogrdvo3TSJKQnJeG/rl/H+JQUvPz448hKTgYAjH7mGSTyt9GOaIwPRVVybi7GL16Mq/v2ITEQwP+ZOBHfnThR/+FDg8GAtPx8WBcu5JHPCMf4UFQZjEaMe/ZZAIDnP/8T3X//Owyfv3vdkJQE7cknMWH5cpg+/xXcNHIxPhR1xqQkZBUXwzJrFnznzsHvdsOYkoK0qVORNnUqElJSVI9IUcD4kBIGoxHJX/kKkr/yFdWjkCJ8Uk1ESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKQE40NESjA+RKRERPGprKzEjBkzoGkaNE2D3W7HkSNH9P1dXV1wOp0YM2YM0tLSUFJSAo/HE/Y9WltbUVxcjFGjRiErKwvr169HT0/P0NwbIoobEcVnwoQJ2Lp1KxobG3HmzBl84xvfwIsvvogLFy4AANatW4dDhw6huroa9fX1uHbtGhYtWqRfPxgMori4GIFAACdOnMC+ffuwd+9ebN68eWjvFRHFPhmk0aNHyxtvvCEdHR2SlJQk1dXV+r5Lly4JAHG5XCIicvjwYTEajeJ2u/U1lZWVomma+P3+L7yNrq4u8Xq9+qWtrU0AiNfrHez4RDSEvF5vvx+bA37NJxgMoqqqCnfu3IHdbkdjYyO6u7vhcDj0NVOmTEFubi5cLhcAwOVyYfr06bBarfqaoqIi+Hw+/eipLxUVFbBYLPolJydnoGMTUYyIOD7Nzc1IS0uD2WzGihUrcODAAeTn58PtdsNkMiEjIyNsvdVqhdvtBgC43e6w8PTu7933RcrLy+H1evVLW1tbpGMTUYyJ+DOcn3jiCTQ1NcHr9eKtt95CaWkp6uvrh2M2ndlshvme33RJRPEv4viYTCZMmjQJAFBQUIDTp0/j9ddfx0svvYRAIICOjo6wox+PxwObzQYAsNlsOHXqVNj36z0b1ruGiB4Ng/45n1AoBL/fj4KCAiQlJaGurk7f19LSgtbWVtjtdgCA3W5Hc3Mz2tvb9TW1tbXQNA35+fmDHYWI4khERz7l5eV47rnnkJubi87OTuzfvx8ffPAB3n//fVgsFixfvhxlZWXIzMyEpmlYvXo17HY75s2bBwBYsGAB8vPzsXTpUmzbtg1utxsbN26E0+nk0yqiR0xE8Wlvb8f3v/99XL9+HRaLBTNmzMD777+Pb37zmwCA7du3w2g0oqSkBH6/H0VFRdi1a5d+/YSEBNTU1GDlypWw2+1ITU1FaWkptmzZMrT3iohinkHk899VG0d8Ph8sFgu8Xi80TVM9DhF9LpLHJt/bRURKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKMD5EpATjQ0RKDCo+W7duhcFgwNq1a/VtXV1dcDqdGDNmDNLS0lBSUgKPxxN2vdbWVhQXF2PUqFHIysrC+vXr0dPTM5hRiCjODDg+p0+fxm9/+1vMmDEjbPu6detw6NAhVFdXo76+HteuXcOiRYv0/cFgEMXFxQgEAjhx4gT27duHvXv3YvPmzQO/F0QUf2QAOjs7ZfLkyVJbWyvPPPOMrFmzRkREOjo6JCkpSaqrq/W1ly5dEgDicrlEROTw4cNiNBrF7XbrayorK0XTNPH7/X3eXldXl3i9Xv3S1tYmAMTr9Q5kfCIaJl6vt9+PzQEd+TidThQXF8PhcIRtb2xsRHd3d9j2KVOmIDc3Fy6XCwDgcrkwffp0WK1WfU1RURF8Ph8uXLjQ5+1VVFTAYrHol5ycnIGMTUQxJOL4VFVV4ezZs6ioqHhgn9vthslkQkZGRth2q9UKt9utr7k3PL37e/f1pby8HF6vV7+0tbVFOjYRxZjESBa3tbVhzZo1qK2tRXJy8nDN9ACz2Qyz2Ry12yOi4RfRkU9jYyPa29sxa9YsJCYmIjExEfX19dixYwcSExNhtVoRCATQ0dERdj2PxwObzQYAsNlsD5z96v26dw0RjXwRxWf+/Plobm5GU1OTfpk9ezaWLFmi/zkpKQl1dXX6dVpaWtDa2gq73Q4AsNvtaG5uRnt7u76mtrYWmqYhPz9/iO4WEcW6iJ52paenY9q0aWHbUlNTMWbMGH378uXLUVZWhszMTGiahtWrV8Nut2PevHkAgAULFiA/Px9Lly7Ftm3b4Ha7sXHjRjidTj61InqERBSf/ti+fTuMRiNKSkrg9/tRVFSEXbt26fsTEhJQU1ODlStXwm63IzU1FaWlpdiyZctQj0JEMcwgIqJ6iEj5fD5YLBZ4vV5omqZ6HCL6XCSPTb63i4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUYHyISAnGh4iUSFQ9wECICADA5/MpnoSI7tX7mOx9jD5MXMbn5s2bAICcnBzFkxBRXzo7O2GxWB66Ji7jk5mZCQBobW390jsYa3w+H3JyctDW1gZN01SP02+cO7ridW4RQWdnJ7Kzs790bVzGx2j87KUqi8USV/9h7qVpWlzOzrmjKx7n7u8BAV9wJiIlGB8iUiIu42M2m/Hqq6/CbDarHiVi8To7546ueJ07EgbpzzkxIqIhFpdHPkQU/xgfIlKC8SEiJRgfIlKC8SEiJeIyPjt37sTEiRORnJyMwsJCnDp1Suk8x48fxwsvvIDs7GwYDAYcPHgwbL+IYPPmzRg/fjxSUlLgcDjw8ccfh625desWlixZAk3TkJGRgeXLl+P27dvDOndFRQXmzJmD9PR0ZGVlYeHChWhpaQlb09XVBafTiTFjxiAtLQ0lJSXweDxha1pbW1FcXIxRo0YhKysL69evR09Pz7DNXVlZiRkzZug//Wu323HkyJGYnrkvW7duhcFgwNq1a+Nu9iEhcaaqqkpMJpP87ne/kwsXLsjLL78sGRkZ4vF4lM10+PBh+dnPfiZvv/22AJADBw6E7d+6datYLBY5ePCg/PnPf5Zvf/vbkpeXJ3fv3tXXPPvsszJz5kw5efKk/OlPf5JJkybJ4sWLh3XuoqIi2bNnj5w/f16amprk+eefl9zcXLl9+7a+ZsWKFZKTkyN1dXVy5swZmTdvnnzta1/T9/f09Mi0adPE4XDIuXPn5PDhwzJ27FgpLy8ftrn/8Ic/yLvvvit/+ctfpKWlRX76059KUlKSnD9/PmZnvt+pU6dk4sSJMmPGDFmzZo2+PR5mHypxF5+5c+eK0+nUvw4Gg5KdnS0VFRUKp/qn++MTCoXEZrPJa6+9pm/r6OgQs9ksb775poiIXLx4UQDI6dOn9TVHjhwRg8EgV69ejdrs7e3tAkDq6+v1OZOSkqS6ulpfc+nSJQEgLpdLRD4Lr9FoFLfbra+prKwUTdPE7/dHbfbRo0fLG2+8ERczd3Z2yuTJk6W2tlaeeeYZPT7xMPtQiqunXYFAAI2NjXA4HPo2o9EIh8MBl8ulcLIvduXKFbjd7rCZLRYLCgsL9ZldLhcyMjIwe/ZsfY3D4YDRaERDQ0PUZvV6vQD++akBjY2N6O7uDpt9ypQpyM3NDZt9+vTpsFqt+pqioiL4fD5cuHBh2GcOBoOoqqrCnTt3YLfb42Jmp9OJ4uLisBmB+Pj7Hkpx9a72Tz/9FMFgMOwvHgCsVisuX76saKqHc7vdANDnzL373G43srKywvYnJiYiMzNTXzPcQqEQ1q5di6eeegrTpk3T5zKZTMjIyHjo7H3dt959w6W5uRl2ux1dXV1IS0vDgQMHkJ+fj6amppidGQCqqqpw9uxZnD59+oF9sfz3PRziKj40fJxOJ86fP48PP/xQ9Sj98sQTT6CpqQlerxdvvfUWSktLUV9fr3qsh2pra8OaNWtQW1uL5ORk1eMoF1dPu8aOHYuEhIQHXv33eDyw2WyKpnq43rkeNrPNZkN7e3vY/p6eHty6dSsq92vVqlWoqanBsWPHMGHCBH27zWZDIBBAR0fHQ2fv67717hsuJpMJkyZNQkFBASoqKjBz5ky8/vrrMT1zY2Mj2tvbMWvWLCQmJiIxMRH19fXYsWMHEhMTYbVaY3b24RBX8TGZTCgoKEBdXZ2+LRQKoa6uDna7XeFkXywvLw82my1sZp/Ph4aGBn1mu92Ojo4ONDY26muOHj2KUCiEwsLCYZtNRLBq1SocOHAAR48eRV5eXtj+goICJCUlhc3e0tKC1tbWsNmbm5vD4llbWwtN05Cfnz9ss98vFArB7/fH9Mzz589Hc3Mzmpqa9Mvs2bOxZMkS/c+xOvuwUP2Kd6SqqqrEbDbL3r175eLFi/LKK69IRkZG2Kv/0dbZ2Snnzp2Tc+fOCQD55S9/KefOnZNPPvlERD471Z6RkSHvvPOOfPTRR/Liiy/2ear9ySeflIaGBvnwww9l8uTJw36qfeXKlWKxWOSDDz6Q69ev65d//OMf+poVK1ZIbm6uHD16VM6cOSN2u13sdru+v/fU74IFC6SpqUnee+89GTdu3LCe+t2wYYPU19fLlStX5KOPPpINGzaIwWCQP/7xjzE78xe592xXvM0+WHEXHxGRX//615Kbmysmk0nmzp0rJ0+eVDrPsWPHBMADl9LSUhH57HT7pk2bxGq1itlslvnz50tLS0vY97h586YsXrxY0tLSRNM0WbZsmXR2dg7r3H3NDED27Nmjr7l796786Ec/ktGjR8uoUaPkO9/5jly/fj3s+/ztb3+T5557TlJSUmTs2LHy4x//WLq7u4dt7h/+8Ify2GOPiclkknHjxsn8+fP18MTqzF/k/vjE0+yDxc/zISIl4uo1HyIaORgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJRgfIlKC8SEiJf4/bjn9kMl5I0sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-790.1836060086542 200 [-7.668710652582663, -7.749618011644003, -7.832646254313315, -7.921582247022011, -8.003297012144314, -8.077874110724991, -8.138879355754458, -8.174623073417326, -8.185969080012654, -8.172693743584139, -8.135336800842172, -8.075243444865041, -7.994511617168069, -7.895918790474709, -7.782829853259477, -7.659087555225119, -7.528887340017541, -7.396639297729064, -7.26682123210802, -7.143828101645566, -7.031823942575098, -6.934602488533198, -6.855461965059504, -6.797098135017774, -6.761518003848353, -6.749975141379802, -6.76292672826225, -6.800012352048866, -6.860055140765672, -6.940574601792137, -7.046412540429422, -7.17526720178951, -7.335488503377806, -7.5370220590869454, -7.790995238546467, -8.136133868918487, -8.620262329937702, -9.324069077448666, -10.18820889170464, -9.946653556031766, -9.229233150624875, -8.465572850169128, -7.6803721318014535, -6.901019519229578, -6.15535633710814, -5.468933875357536, -4.862706300182905, -4.351861004713439, -3.9459544357296754, -3.6500439653947434, -3.4662853646328453, -3.3954880621735906, -3.4382644044732436, -3.5955624022224613, -3.8684851494372223, -4.257383979227022, -4.760307054835037, -5.37103036691801, -6.077099247883329, -6.858502760671014, -7.687651814932585, -8.531080215429856, -9.350259050715218, -10.2163222614294, -10.434927131761976, -9.559301045603192, -8.63141292496528, -7.682078979715462, -6.746338851384875, -5.85936504083393, -5.05177069284245, -4.3461838601999325, -3.7561297609220556, -3.287106491850184, -2.9390023528098004, -2.7088854654409644, -2.593466676295839, -2.5908826293384934, -2.701684791194984, -2.929015062439197, -3.277945069176583, -3.7539304304596284, -4.360335865584199, -5.09230186263472, -6.20091369039514, -7.7699621867245, -9.726336472833163, -11.929851876208005, -12.528425322539535, -11.022224223402302, -9.440669068328448, -7.642173026188523, -5.770928609815662, -4.47385733833554, -3.549974276682413, -2.7911487464086773, -2.1851777205497687, -1.7116379677600846, -1.3474678216487315, -1.0705731204708415, -0.8616635776545981, -0.7048635561824629, -0.5876200054820401, -0.5002841573888218, -0.43559526401020643, -0.38818553478525925, -0.35415759212511627, -0.33074871228212516, -0.3160785980279164, -0.30897105564659344, -0.30883940436206736, -0.31562770292612213, -0.32980336382307873, -0.35240069727589235, -0.3851190389804301, -0.43048310325208783, -0.4920765927900712, -0.5748618370783618, -0.6855961574694751, -0.8333456913123849, -1.0300725364296528, -1.2912204845012127, -1.6361343120343919, -2.088005882810187, -2.6728543231055744, -3.416878722512853, -4.341538532344609, -5.452364670474303, -7.030119509303428, -8.645208835867317, -10.245461559674203, -11.818846654332988, -12.089244629433344, -10.465499399233865, -9.11928706525043, -7.76082348830908, -6.453446558631715, -5.254958183779934, -4.205356546529258, -3.32220826817669, -2.6033319764130876, -2.0332567772849863, -1.5896895915248956, -1.2468149620141518, -0.9898501782146729, -0.7956275844704281, -0.6490825002435637, -0.5383875000526032, -0.45447004453248124, -0.3904770062463625, -0.34128093994968256, -0.3030646102081718, -0.27298958107907356, -0.24894074767142504, -0.22933394893543102, -0.21297359108050035, -0.1989489397898267, -0.18656008903721305, -0.1752669947036338, -0.16465717619603978, -0.15442975085338748, -0.14439550168594434, -0.13449489701112446, -0.12483867102851705, -0.11577913856542255, -0.10802542079879869, -0.10213504112917537, -0.09616817789294434, -0.08377500492409702, -0.06826150198296893, -0.04873623644403276, -0.03233609773007975, -0.021046060565000517, -0.0122274915503713, -0.007968441401784985, -0.005190988878207995, -0.0035700227514502817, -0.0010981002299608814, -0.0009092110328009378, -0.00101564986314577, -0.0006121541952440779, -0.00030207249421703456, -0.0002726965584526197, -0.0004406254798147084, -0.00024227335620224643, -0.00036347267278441845, -0.0004152701506383153, -0.00029169070966805836, -0.0003241118770540448, -0.0005565294184578679, -0.0005448178054003114, -0.0004383884675636466, -0.0005179247321218769, -0.00045003265004908385, -0.0004897515816138963, -0.0004696844372904799, -0.000521892856330205, -0.0004556934572858486, -0.00047151267109808145, -0.00044596139975303626]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
